import pymysql
import requests
import json
import io
import sys

# 腾讯新闻热搜后端传输api，下面地址为JS请求的数据接口，f12从network/xhr中获取
start_url = 'https://r.inews.qq.com/getWeiboRankingList?chlid=news_recommend_hot&appver=28_android_6.8.00&devid=&qn-rid=&qn-sig=63b186047dc02809a9ec82e5c3ded91f&is_h5=1'
# 定义要插入数据的表名和数据库名
tablename='t_txhot'
databasename='resoudata'
# 定义浏览器标识和相应cookie，F12+network获取
# 添加请求头
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.55 Safari/537.36 Edg/96.0.1054.43',
    'cookie': 'RK=apUA3+pEXJ; ptcz=c51da3bfb1297f2bd404c184cf85b6516f2f724ee0ab6b4d8f856f16f00628ea; pgv_pvid=8083814540; pac_uid=0_62a1b4e22dac8; fqm_pvqid=9fd01dae-2c07-4e25-a2a3-292a1867220e; tvfe_boss_uuid=fc58a14c8b7e5683; ariaDefaultTheme=undefined'
}

# 改变标准输出的默认编码，防止spring boot运行时显示乱码
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8')

# 保存到数据库
def saveDAO(db,number,topic,hot,href):
    cursor = db.cursor()
    query = """insert into """+ tablename +""" (ranking,topic,hot,url,daydate,mindate) values (%s,%s,%s,%s,curdate(),curtime())"""
    cursor.execute(query,(number,topic,hot,href))
    cursor.close()

# 进行requests请求
def get(url):
    response = requests.get(url, headers=headers)
    if response. status_code == 200:
        print("腾讯新闻热搜请求成功")
        parse(url)
    else:
        print("网页请求失败了")

# 将今日头条热搜的数据给爬下来
def parse(url):
    # 因为id自动递增，让自动增长从1开始，已经有了就跳过，让id连续
    db = pymysql.connect(host="localhost", user="root", password="root", database=databasename)
    cursor = db.cursor()
    str = """ALTER TABLE """ + tablename  + """ AUTO_INCREMENT = 1"""
    cursor.execute(str)
    cursor.close()
    db.commit()

    # 由于是纯json数据，将其用文本形式保存
    response = requests.get(url, headers=headers)
    data = json.loads(response.text)
    # 获取json数据
    hotdata = data['idlist'][0]['newslist']
    hotdata.pop(0)
    try:
        for i in hotdata:
            number=i['hotEvent']['ranking']
            topic = i['hotEvent']['title']
            hot = i['hotEvent']['hotScore']
            href = "https://view.inews.qq.com/a/"+i['id']+"?uid="
            # print(number,topic, hot, href)
            saveDAO(db, number, topic, hot, href)
    except BaseException as ex:
        print(ex)
    else:
        db.commit()
        db.close()
    response.close()

#程序从此处开始运行
if __name__ == "__main__":
    get(start_url)
